import abc
import copy
import scrapy
from scrapy import Request, FormRequest


class ZhengFuBaseSpider(scrapy.Spider):
    """最基础的政府蜘蛛"""
    name = ''
    allowed_domains = ['']
    start_urls = ['']
    # 关键字
    keywords = ['碳普惠',
            '碳评价',
            '碳惠天府',
            '碳核准',
            '碳积分',
            '低碳产品平台',
            '低碳供应链制度',
            '碳排放控制制度',
            '碳金融',
            '低碳绩效评价',
            '低碳管理',
            '碳汇生态',
            '低碳推广',
            '碳准入',
            '低碳农业发展',
            '低碳产业扶贫',
            '碳合一',
            '碳管理',
            '低碳发展机制',
            '森林碳汇',
            '低碳管理平台',
            '低碳示范企业',
            '碳信息平台',
            '光伏扶贫',
            '低碳宣传',
            '低碳特色产品',
            '低碳制度创新',
            '碳排放权',
            '低碳技术'
            '低碳产品',
            '碳达峰追踪制度',
            '低碳技术创新',
            '低碳优惠政策',
            '碳管理平台',
            '低碳创新机制',
            '低碳特色产业',
            '低碳协同',
            '绿色推广',
            '低碳咨询',
            '低碳考评机制',
            '低碳考核机制',
            '低碳示范工程',
            '低碳交通',
            '碳监督',
            '碳预警',
            '低碳社区',
            '低碳激励机制',
            '低碳认证技术标准',
            '碳排放监测',
            '低碳多领域',
            '低碳目标责任评估',
            '低碳立法支持',
            '碳标识',
            '低碳科技创新',
            '绿色金融',
            '低碳转型示范'
            ]
    # API
    api = ""
    # 模式 GET/POST
    method = "default"
    # 测试用 headers
    headers = {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:94.0) Gecko/20100101 Firefox/94.0",
        "Accept": "*/*",
        "Accept-Language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
        "Accept-Encoding": "gzip, deflate, br",
        "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8"
    }
    # 数据模板
    data = {}
    # 是否解析第一页
    parse_first = True
    # 起始页的索引 (某些情况下需要调整为 0 )
    start_page = 1

    def start_requests(self):
        """
        抛出初始请求
        """
        self.logger.debug("check keywords")
        keywords = self.keywords
        if not keywords:
            raise Exception("Need keywords!")
        self.logger.debug(f"keywords: {keywords}")

        self.logger.debug("check API")
        api = self.api
        if not api:
            raise Exception("Need API!")

        self.logger.debug("check method")
        method = self.method
        if not method:
            raise Exception("Need method!")

        if method == "GET":
            yield from self.start_get_requests(
                page=self.start_page,
                callback=self.parse_index
            )
        elif method == "POST":
            yield from self.start_post_requests(
                page=self.start_page,
                callback=self.parse_index
            )
        else:
            raise Exception("Invalid method!")

    def start_get_requests(self, page=None, callback=None):
        """抛出起始GET请求"""
        keywords = self.keywords
        api = self.api
        headers = self.headers
        for keyword in keywords:
            url = api.format(keyword=keyword, page=page)
            req = Request(
                url=url,
                meta={"keyword": keyword},
                headers=headers,
                callback=callback,
                dont_filter=True
                )
            yield req

    def start_post_requests(self, page=None, callback=None):
        """抛出起始POST请求"""
        keywords = self.keywords
        url = self.api
        headers = self.headers
        self.logger.debug(f"爬取 第{page}页")
        for keyword in keywords:
            data = self.build_data(keyword, page)
            self.logger.debug(f"Post 数据: {data}")
            req = FormRequest(
                url=url,
                meta={"keyword": keyword},
                headers=headers,
                formdata=data,
                callback=callback,
                dont_filter=True
                )
            yield req

    def parse_index(self, response):
        start_page = self.start_page
        # 获取总页数
        total_page = self.edit_page(response)
        self.logger.debug(f"总页数: {total_page}")
        end_page = start_page + total_page
        # 重新抛出请求
        if self.method == "GET":
            for page in range(start_page, end_page):
                yield from self.start_get_requests(
                    page=page,
                    callback=self.parse_items,
                )
        elif self.method == "POST":
            for page in range(start_page, end_page):
                yield from self.start_post_requests(
                    page=page,
                    callback=self.parse_items
                )

    def build_data(self, keyword, page):
        data = copy.copy(self.data)
        data = self.edit_data(data, keyword, page)
        return data

    def parse_items(self, response):
        items_box = self.edit_items_box(response)
        if not items_box:
            self.logger.debug("No items_box")
            return None
        items = self.edit_items(items_box)
        keyword = response.meta["keyword"]
        for item in items:
            yield self.parse_item(item, keyword)

    def parse_item(self, item, keyword):
        item = self.edit_item(item)
        item["keyword"] = keyword
        if not item["url"]:
            self.logger.debug("No url")
            return None
        # yield item
        return Request(
            url=item["url"],
            headers=self.headers,
            meta={"item": item},
            callback=self.parse_article
        )

    def parse_article(self, response):
        item = response.meta["item"]
        article = self.edit_article(response)
        article |= item
        return article

    @abc.abstractmethod
    def edit_data(self, data, keyword, page):
        """
        input:
        data:dict
        return:
        data:dict
        """
        raise Exception("Must rewrite edit_data")

    @abc.abstractmethod
    def edit_items_box(self, response):
        """
        input: response
        return: items_box
        """
        raise Exception("Must rewrite edit_items_box")

    @abc.abstractmethod
    def edit_items(self, items_box):
        """
        input: items_box
        return: items
        """
        return items_box

    @abc.abstractmethod
    def edit_item(self, item):
        """
        input: item
        return: item_dict
        """
        return item

    @abc.abstractmethod
    def edit_page(self, response):
        """
        input: response
        return: int
        """
        raise Exception("Must rewrite edit_page")

    @abc.abstractmethod
    def edit_article(self, response):
        """
        input: response
        return: item
        """
        raise Exception("Must rewrite edit_page")
